/*
  author Sylvain Bertrand <sylvain.bertrand@gmail.com>
  Protected by linux GNU GPLv2
  Copyright 2012-2014
*/
#include <linux/pci.h>
#include <asm/byteorder.h>
#include <linux/interrupt.h>
#include <linux/firmware.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/cdev.h>
#include <linux/fs.h>
#include <linux/idr.h>
#include <linux/pm.h>
#include <linux/list.h>

#include <alga/alga.h>
#include <alga/rng_mng.h>
#include <uapi/alga/pixel_fmts.h>
#include <alga/timing.h>
#include <alga/amd/atombios/atb.h>
#include <alga/amd/atombios/pp.h>
#include <uapi/alga/amd/dce6/dce6.h>

#define DRV_C

#include "mc.h"
#include "rlc.h"
#include "ih.h"
#include "fence.h"
#include "ring.h"
#include "dmas.h"
#include "ba.h"
#include "cps.h"
#include "gpu.h"
#include "drv.h"

#include "fops/fops.h"
#include "intr_irq.h"
#include "hdp.h"
#include "bif.h"
#include "smc.h"
#include "golden.h"
#include "vm.h"
#include "cm.h"
#include "dyn_pm/dyn_pm.h"

#include "regs.h"

bool no_dpm = 0;
module_param(no_dpm, bool, 0);
MODULE_PARM_DESC(no_dpm, "disable dynamic power management (end in power boot state which is not the highest)");

#define GPUS_MAX 256
static DEFINE_IDA(ida);

static struct class *class;

static struct atb_dev adev;
static struct dce6_dev ddev;

/* those functions exist to deal with regs above regs range */
u32 rr32(struct pci_dev *dev, u32 aligned_of)
{
	struct dev_drv_data *dd;
	u32 val;

	dd = pci_get_drvdata(dev);

	if (aligned_of <= (dd->regs_sz - sizeof(u32))) {
		val = readl(dd->regs + aligned_of);
	} else {
		/* FIXME: should have a spinlock to make it atomic */
		writel((u32)(aligned_of >> 31), dd->regs + MM_IDX_HI);
		writel(lower_32_bits(aligned_of) & ~MI_VRAM, dd->regs + MM_IDX);
		val = readl(dd->regs + MM_DATA);
	}
	return val;
}

u32 vram_r32(struct pci_dev *dev, u64 aligned_of)
{
	struct dev_drv_data *dd;
	dd = pci_get_drvdata(dev);

	writel(aligned_of >> 31, dd->regs + MM_IDX_HI);
	writel(lower_32_bits(aligned_of) | MI_VRAM, dd->regs + MM_IDX);
	
	return readl(dd->regs + MM_DATA);
}

void wr32(struct pci_dev *dev, u32 val, u32 aligned_of)
{
	struct dev_drv_data *dd;
	dd = pci_get_drvdata(dev);

	if (aligned_of <= (dd->regs_sz - sizeof(u32))) {
		writel(val, dd->regs + aligned_of);
	} else {
		/* FIXME: should have a spinlock to make it atomic */
		writel((u32)(aligned_of >> 31), dd->regs + MM_IDX_HI);
		writel(lower_32_bits(aligned_of) & ~MI_VRAM, dd->regs + MM_IDX);
		writel(val, dd->regs + MM_DATA);	
	}
}

void vram_w32(struct pci_dev *dev, u32 val, u64 aligned_of)
{
	struct dev_drv_data *dd;
	dd = pci_get_drvdata(dev);

	writel((u32)(aligned_of >> 31), dd->regs + MM_IDX_HI);
	writel(lower_32_bits(aligned_of) | MI_VRAM, dd->regs + MM_IDX);
	writel(val, dd->regs + MM_DATA);	
	mb();
}

static u32 extern_rr32(struct device *dev, u32 of)
{
	struct dev_drv_data *dd;
	struct pci_dev *pdev;

	pdev = container_of(dev, struct pci_dev, dev);
	dd = pci_get_drvdata(pdev);

	return rr32(pdev, of);
}

static void extern_wr32(struct device *dev, u32 val, u32 of)
{
	struct dev_drv_data *dd;
	struct pci_dev *pdev;

	pdev = container_of(dev, struct pci_dev, dev);
	dd = pci_get_drvdata(pdev);

	wr32(pdev, val, of);
}

static void *rom_copy_get(struct pci_dev *dev)
{
	void __iomem *rom;
	void *rom_copy;
	size_t rom_sz;

	rom = pci_map_rom(dev, &rom_sz);
	if (IS_ERR_OR_NULL(rom))
		return rom;

	rom_copy = kzalloc(rom_sz, GFP_KERNEL);
	if (IS_ERR_OR_NULL(rom_copy)) {
		pci_unmap_rom(dev, rom);
		return ERR_PTR(-ENOMEM);
	}

	memcpy_fromio(rom_copy, rom, rom_sz);
	pci_unmap_rom(dev, rom);
	return rom_copy;
}

static void bif_mc_mem_reqs_dis(struct pci_dev *dev)
{
	u32 m;

	m = mc_clients_blackout_mode(dev);

	if (m != MSBC_MODE_ENA) {
		bif_hdp_dis(dev);
		mc_clients_blackout_ena(dev);
	}
}

static void bif_mc_mem_reqs_ena(struct pci_dev *dev)
{
	mc_clients_blackout_dis(dev);
	bif_hdp_ena(dev);
}

static long regs_bar_map(struct pci_dev *dev)
{
	struct dev_drv_data *dd;
	dd = pci_get_drvdata(dev);

	dd->regs = ioremap_nocache(pci_resource_start(dev, 2),
						pci_resource_len(dev, 2));
	if (!dd->regs)
		return -SI_ERR;
	dd->regs_sz = pci_resource_len(dev, 2);
	
	dev_info(&dev->dev, "regs mmio base: 0x%p\n", dd->regs);
	dev_info(&dev->dev, "regs mmio size: %zu\n",
					(size_t)pci_resource_len(dev, 2));
	return 0;
}

static void cfg_init(struct pci_dev *dev)
{
	struct dev_drv_data *dd;

	dd = pci_get_drvdata(dev);

	switch (dd->family) {
	case TAHITI:
		dd->cfg.addr_best = 0x12011003;
		dd->cfg.dce_crtcs_n = 6;
		break;
	case PITCAIRN:
		dd->cfg.addr_best = 0x12011003;
		dd->cfg.dce_crtcs_n = 6;
		break;
	case VERDE:
		dd->cfg.addr_best = 0x1205002;
		dd->cfg.dce_crtcs_n = 6;
		break;
	case OLAND:
		dd->cfg.addr_best = 0x1205002;
		dd->cfg.dce_crtcs_n = 2;
	}
	gpu_cfg_init(dev);
}

static ssize_t discret_vram_show(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct dev_drv_data *dd;

	dd = dev_get_drvdata(dev);

	return scnprintf(buf, PAGE_SIZE, "%llu", dd->vram.mng.sz);
}

/*----------------------------------------------------------------------------*/
static DEVICE_ATTR(discret_vram, S_IRUGO, discret_vram_show, NULL);

static struct attribute *char_dev_attrs[] = {
	&dev_attr_discret_vram.attr,
	NULL
};
static struct attribute_group char_dev_attr_group = {
	.attrs = &char_dev_attrs[0]
};
static const struct attribute_group *char_dev_attr_groups[] = {
	&char_dev_attr_group,
	NULL
};
/*----------------------------------------------------------------------------*/

static void dev_release_empty(struct device *dev)
{
}

static long char_dev_register(struct dev_drv_data *dd)
{
	long r;

	memset(&dd->char_dev, 0, sizeof(dd->char_dev));

	dd->char_dev.devt = dd->char_cdev.dev;
	dd->char_dev.class = class;
	dd->char_dev.parent = &dd->dev->dev;
	dd->char_dev.release = dev_release_empty;
	dev_set_drvdata(&dd->char_dev, dd);

	sysfs_attr_init(&dev_attr_discret_vram.attr);
	dd->char_dev.groups = &char_dev_attr_groups[0];

	r = dev_set_name(&dd->char_dev, "si%u", dd->minor);
	if (r != 0) {
		dev_err(&dd->dev->dev, "unable to set char device name\n");
		goto err_put_dev;
	}

	r = device_register(&dd->char_dev);
	if (r != 0) {
		dev_err(&dd->dev->dev, "unable to register char device\n");
		goto err_put_dev;
	}
	return 0;

err_put_dev:
	put_device(&dd->char_dev);
	return -SI_ERR;
}

static void asic_deinit(struct pci_dev *dev)
{
	struct dev_drv_data *dd;

	dd = pci_get_drvdata(dev);

	if (!no_dpm)
		dyn_pm_dis(dev);
	dmas_stop(dev);
	cps_engines_stop(dev);
	if (!no_dpm) {
		vg_dis(dev);
		cg_dis(dev);
	}
	ih_dis(dev);
	ih_reset(dev);
	mdelay(1);
	dce6_irqs_ack(dd->dce);
	rlc_shutdown(dev);
	gpu_3d_ring_intr_idle_dis(dev);
	rlc_serdes_wait(dev);
	rlc_cleanup(dev);
	ba_all_unmap(dev,0);
	ba_shutdown(dev);
	bif_mc_mem_reqs_dis(dev);
	rng_free(&dd->vram.mng, dd->vram.scratch_page);
	/* TODO: should post again the board? */
}

static long asic_init(struct pci_dev *dev)
{
	struct dev_drv_data *dd;
	long r;

	dd = pci_get_drvdata(dev);

	/* posting will ensure the chip is in good shape */
	dev_info(&dev->dev, "posting now...\n");

	r = atb_asic_init(dd->atb, dd->pp.default_eng_clk,
							dd->pp.default_mem_clk);
	if (r == -ATB_ERR) {
		dev_err(&dev->dev, "atombios failed to init the asic\n");
		goto err;
	}
	wmb();
	golden_regs(dev);
	wmb();

	/*
	 * CFG_MEM_SZ is now valid but some boards have garbage in upper 16 bits
	 */
	dev_info(&dev->dev, "vram size is %uMB\n", rr32(dev, CFG_MEM_SZ)
								& 0xffff);

	if (!no_dpm) {
		r = dyn_pm_ena(dev);
		if (r == -SI_ERR)
			goto err;
	}

	/* claim back the 256k vga memory at vram beginning */
	dce6_vga_off(dd->dce);	

	mc_addr_cfg_compute(dev);

	/* some boards have garbage in upper 16 bits */
	rng_mng_init(&dd->vram.mng, 0, (rr32(dev, CFG_MEM_SZ) & 0xffff) * 1024
									* 1024);

	/*--------------------------------------------------------------------*/
	/* quiet the memory requests before mc programming: dce then gpu */
	r = dce6_crtcs_shutdown(dd->dce);
	if (r == -DCE6_ERR)
		goto err_disable_dyn_pm;

	bif_mc_mem_reqs_dis(dev);
	udelay(100);

	if (mc_wait_for_idle(dev)) {
		dev_warn(&dev->dev, "wait for mc idle timed out\n");
		goto err_disable_dyn_pm;
	}
	/*--------------------------------------------------------------------*/

	/*
	 * If the mc is not programmed, upstream says the GPU won't be able to
	 * work anymore. Leave it be since what is inited between
	 * atb_asic_init and mc_program is really not likely to fail, except
	 * if the gpu is already unusable...
	 */
	mc_program(dev);

	hdp_init(dev);

	mc_ucode_program(dev);
	udelay(100);

	r = rng_alloc_align(&dd->vram.scratch_page, &dd->vram.mng,
						GPU_PAGE_SZ, GPU_PAGE_SZ);
	if (r == -ALGA_ERR) {
		dev_err(&dev->dev, "unable to alloc GPU scratch page\n");
		goto err_disable_dyn_pm;
	}

	bif_mc_mem_reqs_ena(dev);

	r = ba_init(dev);	
	if (r == -BA_ERR)
		goto err_free_scratch_page;

	r = ba_core_map(dev); /* map wb page, ih ring, cp rings */
	if (r == -BA_ERR)
		goto err_ba_shutdown;

	/*
	 * XXX:diverging from atombios default state seems to makes the chip
	 * really unstable (sequence of driver module load/unload).
	 */
	/* gpu_defaults(dev); */

	r = rlc_init(dev);
	if (r == -SI_ERR) {
		dev_err(&dev->dev, "failed to prepare the gpu buffers for the rlc\n");
		goto err_ba_core_unmap;
	}

	ih_dis(dev);
	ih_reset(dev);
	rlc_shutdown(dev);
	gpu_3d_ring_intr_idle_dis(dev);
	rlc_serdes_wait(dev);
	rlc_reset(dev);
	if (!no_dpm) {
		vg_ena(dev);/* currently empty */
		cg_ena(dev);
	}
	rlc_ucode_program(dev);
	if (!no_dpm) {
		if (mc_lb_pwr_supported(dev)) {
			rlc_lb_pw_ena(dev);
		} else {
			rlc_lb_pw_dis(dev);
			gpu_lb_pw_dis(dev);
		}
	}
	rlc_ena(dev);
	gpu_3d_ring_intr_idle_ena(dev);
	udelay(50);

	ih_init(dev);
	intrs_reset(dev);
	ih_ena(dev);

	/*--------------------------------------------------------------------*/
	cps_engines_ucode_program(dev);
	gpu_3d_ring_intr_idle_dis(dev);
	cps_init(dev);
	cps_me_init(dev);
	cps_enable(dev);
	cps_ctx_clr(dev);
	gpu_3d_ring_intr_idle_ena(dev);
	/*--------------------------------------------------------------------*/

	dmas_resume(dev);

	r = dce6_init(dd->dce, dd->addr_cfg);
	if (r == -DCE6_ERR)
		goto err_stop_dmas;
	return 0;

err_stop_dmas:
	dmas_stop(dev);

	cps_engines_stop(dev);
	if (!no_dpm) {
		vg_dis(dev);
		cg_dis(dev);
	}
	ih_dis(dev);
	ih_reset(dev);
	mdelay(1);
	dce6_irqs_ack(dd->dce);
	intrs_reset(dev);
	rlc_shutdown(dev);
	gpu_3d_ring_intr_idle_dis(dev);
	rlc_serdes_wait(dev);
	rlc_cleanup(dev);

err_ba_core_unmap:
	ba_all_unmap(dev,0);

err_ba_shutdown:
	ba_shutdown(dev);
	bif_mc_mem_reqs_dis(dev);

err_free_scratch_page:
	rng_free(&dd->vram.mng, dd->vram.scratch_page);

err_disable_dyn_pm:

err:
	return -SI_ERR;
}

static int probe(struct pci_dev *dev, const struct pci_device_id *id)
{
	int r;
	struct dev_drv_data *dd;
	struct atb_pp_defaults atb_pp_defaults;

	r = pci_enable_device(dev);
	if (r) {
		dev_err(&dev->dev, "cannot enable the device\n");
		goto err;
	}

	r = pci_enable_msi(dev);
	if (r) {
		dev_err(&dev->dev, "cannot enable MSI\n");
		goto err_dis_pdev;
	}

	/*--------------------------------------------------------------------*/
	r = dma_set_coherent_mask(&dev->dev, DMA_BIT_MASK(40));
	if (r) {
		dev_err(&dev->dev, "cannot set coherent DMA address width to 40 bits\n");
		goto err_dis_msi;
	}
	/*--------------------------------------------------------------------*/

	r = pci_request_regions(dev, pci_name(dev));
	if (r != 0) {
		dev_err(&dev->dev, "cannot request PCI regions\n");
		goto err_dis_msi;
	}

	/*--------------------------------------------------------------------*/
	dd = kzalloc(sizeof(*dd), GFP_KERNEL);
	if (!dd) {
		dev_err(&dev->dev,
			"cannot allocate driver private data for device\n");	
		goto err_release_pci_regions;
	}
	pci_set_drvdata(dev, dd);
	/*--------------------------------------------------------------------*/

	dd->dev = dev;	/* only used for char device node */
	dd->family = id->driver_data;

	cfg_init(dev);

	adev.rom = rom_copy_get(dev);
	if (IS_ERR_OR_NULL(adev.rom)) {
		dev_err(&dev->dev, "cannot copy the rom\n");
		goto err_free_drv_data;
	}

	r = regs_bar_map(dev);
	if (r == -SI_ERR)
		goto err_free_rom_copy;

	/*--------------------------------------------------------------------*/
	adev.dev = &dev->dev;
	adev.rr32 = extern_rr32;
	adev.wr32 = extern_wr32;

	dd->atb = atb_alloc(&adev);
	if (!dd->atb) {
		dev_err(&dev->dev, "cannot allocate atombios\n");
		goto err_regs_bar_unmap;
	}
	/*--------------------------------------------------------------------*/

	r = atb_init(dd->atb);
	if (r == -ATB_ERR) {
		dev_err(&dev->dev, "cannot init the atombios\n");
		goto err_free_atb;
	}

	/*--------------------------------------------------------------------*/
	ddev.dev = &dev->dev;
	ddev.atb = dd->atb;
	ddev.crtcs_n = dd->cfg.dce_crtcs_n;
	ddev.rr32 = extern_rr32;
	ddev.wr32 = extern_wr32;
	ddev.dyn_pm_new_display_notify = dyn_pm_new_display_notify;

	dd->dce = dce6_alloc(&ddev);

	if (!dd->dce) {
		dev_err(&dev->dev, "cannot allocate dce\n");
		goto err_cleanup_atb;
	}

	r = dce6_init_once(dd->dce);
	if (r == -DCE6_ERR) {
		dev_err(&dev->dev, "unable to do the dce data structures init\n");
		goto err_dce_free;
	}
	/*--------------------------------------------------------------------*/

	/* get defaults clocks and volts */
	r = atb_pp_defaults_get(dd->atb, &atb_pp_defaults);
	if (r == -ATB_ERR) {
		dev_err(&dev->dev, "atombios failed to get default clocks and voltages\n");
		goto err_dce_free;
	}
	dd->pp.default_eng_clk = atb_pp_defaults.eng_clk;
	dd->pp.default_mem_clk = atb_pp_defaults.mem_clk;
	dd->pp.default_vddc = atb_pp_defaults.vddc_mv;
	dd->pp.default_vddci = atb_pp_defaults.vddci_mv;
	dd->pp.default_mvddc = atb_pp_defaults.mvddc_mv;

	r = request_threaded_irq(dev->irq, irq, irq_thd, 0, pci_name(dev),
								(void*)dev);
	if (r) {
		dev_err(&dev->dev, "unable to request threaded irq\n");
		goto err_dce_free;
	}

	r = mc_ucode_load(dev);
	if (r == -SI_ERR)
		goto err_free_irq;

	r = rlc_ucode_load(dev);
	if (r == -SI_ERR)
		goto err_release_mc_fw;

	r = cps_engines_ucode_load(dev);
	if (r == -SI_ERR)
		goto err_release_rlc_fw;

	r = smc_ucode_load(dev);
	if (r == -SI_ERR)
		goto err_release_cps_fw;

	r = ba_init_once(dev);
	if (r == -BA_ERR)
		goto err_release_smc_fw;

	cps_init_once(dev);
	dmas_init_once(dev);

	r = asic_init(dev);
	if (r == -SI_ERR)
		goto err_free_dummy_page;
	dd->asic_inited = 1;

	r = dce6_cold_plug(dd->dce);
	if (r == -DCE6_ERR)
		goto err_deinit_asic;

	/*--------------------------------------------------------------------*/
	/* userland interface setup */
	fops_init_once();

	cdev_init(&dd->char_cdev, &fops);

	r = ida_simple_get(&ida, 0, GPUS_MAX, GFP_KERNEL);
	if (r < 0) {
		if (r == -ENOMEM) {
			dev_err(&dev->dev, "no memory for minor\n");
		} else if (r == -ENOSPC) {
			dev_err(&dev->dev, "no more room for new minor\n");
		} else {
			dev_err(&dev->dev, "minor allocation failed\n");
		}
		goto err_free_dce;
	}
	dd->minor=(unsigned int)r;

	r = cdev_add(&dd->char_cdev, MKDEV(MAJOR(devt_region), dd->minor), 1);
	if (r) {
		dev_err(&dev->dev, "cannot add register char device\n");
		goto err_free_minor;
	}

	r = char_dev_register(dd);
	if (r)
		goto err_cdev_del;

	r = dce6_sysfs_cold_register(dd->dce, &dd->char_dev);
	if (r) {
		goto err_unregister_device;
	}
	/*--------------------------------------------------------------------*/

	pci_set_master(dev);
	dce6_hpds_intr_ena(dd->dce);
	cps_intr_ena(dev);
	dmas_intr_ena(dev);

	dev_info(&dev->dev, "ready\n");
	return 0;

err_unregister_device:
	device_unregister(&dd->char_dev);

err_cdev_del:
	cdev_del(&dd->char_cdev);

err_free_minor:
	ida_simple_remove(&ida, dd->minor);

err_free_dce:
	dce6_off(dd->dce);
	dce6_edid_free(dd->dce);
	dce6_i2c_cleanup(dd->dce);

err_deinit_asic:
	asic_deinit(dev);

err_free_dummy_page:
	dma_free_coherent(&dev->dev, GPU_PAGE_SZ, dd->ba.dummy_cpu_addr,
							dd->ba.dummy_bus_addr);
err_release_smc_fw:
	release_firmware(dd->smc_fw);

err_release_cps_fw:
	release_firmware(dd->pfp_fw);
	release_firmware(dd->ce_fw);
	release_firmware(dd->me_fw);

err_release_rlc_fw:
	release_firmware(dd->rlc_fw);

err_release_mc_fw:
	release_firmware(dd->mc_fw);

err_free_irq:
	free_irq(dev->irq, (void*)dev);

err_dce_free:
	kfree(dd->dce);

err_cleanup_atb:
	atb_cleanup(dd->atb);

err_free_atb:
	kfree(dd->atb);

err_regs_bar_unmap:
	iounmap(dd->regs);

err_free_rom_copy:
	kfree(adev.rom);

err_free_drv_data:
	pci_set_drvdata(dev, NULL);
	kfree(dd);

err_release_pci_regions:
	pci_release_regions(dev);

err_dis_msi:
	pci_disable_msi(dev);

err_dis_pdev:
	pci_disable_device(dev);
err:
	return -SI_ERR;
}

static void remove(struct pci_dev *dev)
{
	struct dev_drv_data *dd;

	dd = pci_get_drvdata(dev);

	pci_clear_master(dev);

	dce6_sysfs_unregister(dd->dce);
	device_unregister(&dd->char_dev);
	cdev_del(&dd->char_cdev);
	ida_simple_remove(&ida, dd->minor);

	dce6_off(dd->dce);
	dce6_edid_free(dd->dce);
	dce6_i2c_cleanup(dd->dce);
	if (dd->asic_inited) {
		dyn_pm_dis(dev);
		asic_deinit(dev);
		dd->asic_inited = 0;
	}

	dma_free_coherent(&dev->dev, GPU_PAGE_SZ, dd->ba.dummy_cpu_addr,
							dd->ba.dummy_bus_addr);
	release_firmware(dd->mc_fw);
	release_firmware(dd->rlc_fw);
	release_firmware(dd->pfp_fw);
	release_firmware(dd->ce_fw);
	release_firmware(dd->me_fw);
	release_firmware(dd->rlc_fw);
	free_irq(dev->irq, (void*)dev);
	kfree(dd->dce);
	rng_mng_destroy(&dd->vram.mng);
	atb_cleanup(dd->atb);
	kfree(dd->atb);
	iounmap(dd->regs);
	kfree(adev.rom);
	pci_release_regions(dev);
	pci_disable_msi(dev);
	pci_disable_device(dev);
	pci_set_drvdata(dev, NULL);
	kfree(dd);
}

static int si_prepare(struct device *dev)
{
	struct pci_dev *pdev;
	struct dev_drv_data *dd;

	pdev = to_pci_dev(dev);
	dd = pci_get_drvdata(pdev);

	dce6_hpd_dev_registration_lock(dd->dce);	

	dce6_sysfs_unregister(dd->dce);
	dce6_edid_free(dd->dce);
	dce6_i2c_cleanup(dd->dce);
	dev_info(dev, "prepare:finished\n");
	return 0;
}

static void si_complete(struct device *dev)
{
	struct pci_dev *pdev;
	struct dev_drv_data *dd;
	long r;

	pdev = to_pci_dev(dev);
	dd = pci_get_drvdata(pdev);

	r = dce6_cold_plug(dd->dce);
	if (r == -DCE6_ERR) 
		dev_err(dev, "complete:dce cold plug failed\n");

	r = dce6_sysfs_cold_register(dd->dce, &dd->char_dev);
	if (r == -DCE6_ERR) 
		dev_err(dev, "complete:dce sysfs cold register failed\n");

	dce6_hpd_dev_registration_unlock(dd->dce);

	dce6_hpds_intr_ena(dd->dce);
	cps_intr_ena(pdev);
	dev_info(dev, "complete:finished\n");
}

static int si_suspend_noirq(struct device *dev)
{
	struct pci_dev *pdev;
	struct dev_drv_data *dd;
	struct file_private_data *file_private_data;

	pdev = to_pci_dev(dev);
	dd = pci_get_drvdata(pdev);

	/* vram content lost during suspend */
	rng_mng_destroy(&dd->vram.mng);

	/*
	 * TODO: wrong, the mapping must still have ram after coming back
	 * from suspend.
	 * old:unmap all aperture maps without pt update since it's useless
	 */
	ba_all_unmap(pdev, BA_NO_PT_UPDATE);

	/* "context is lost to userspace", no concurrent access to list */
	list_for_each_entry(file_private_data, &files_private_data, n) {
		file_private_data->context_lost = 1;
	}
	dev_info(dev, "suspend_noirq:finished\n");
	return 0;
}

static int si_resume_noirq(struct device *dev)
{
	struct pci_dev *pdev;
	struct dev_drv_data *dd;
	long r;

	pdev = to_pci_dev(dev);
	dd = pci_get_drvdata(pdev);

	dd->asic_inited = 0;

	r = asic_init(pdev);
	if (r == -SI_ERR) {
		dev_err(dev, "resume_noirq:failed to init the asic\n");
		return -SI_ERR;
	}

	dd->asic_inited = 1;
	dev_info(dev, "resume_noirq:successfull\n");
	return 0;	
}

/* code path for mobility edition not in this driver */
static DEFINE_PCI_DEVICE_TABLE(pci_tbl) =
{
	{PCI_VENDOR_ID_ATI, 0x6600, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6601, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6602, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6603, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6604, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6605, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6606, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6607, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6608, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6610, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6611, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6613, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6620, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6621, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6623, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6631, PCI_ANY_ID, PCI_ANY_ID, 0, 0, OLAND},
	{PCI_VENDOR_ID_ATI, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6784, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6788, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x678a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6790, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6791, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6792, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6798, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6799, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x679a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x679b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x679e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x679f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, TAHITI},
	{PCI_VENDOR_ID_ATI, 0x6800, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6801, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6802, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6806, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6808, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6809, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6810, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6811, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6816, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6817, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6818, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6819, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{PCI_VENDOR_ID_ATI, 0x6820, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6821, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6822, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6823, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6824, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6825, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6826, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6827, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6828, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6829, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x682a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x682b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x682c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x682d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x682f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x6839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x683b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x683d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x683f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, VERDE},
	{PCI_VENDOR_ID_ATI, 0x684c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PITCAIRN},
	{}
};
MODULE_DEVICE_TABLE(pci, pci_tbl);

static struct dev_pm_ops dev_pm_ops = {
	.prepare = si_prepare,
	.complete = si_complete,
	.suspend_noirq = si_suspend_noirq,
	.resume_noirq = si_resume_noirq
};

static struct pci_driver pci_driver =
{
	.name = "AMD southern island PCI driver",
	.id_table = pci_tbl,
	.probe = probe,
	.remove = remove,
	.driver.pm = &dev_pm_ops
};

static int init(void)
{
	int r;

	ida_init(&ida);

	class = class_create(THIS_MODULE, "si");	
	if (IS_ERR_OR_NULL(class)) {
		if (IS_ERR(class))
			return PTR_ERR(class);
		else
			return -ENOMEM;
	}

	r = alloc_chrdev_region(&devt_region, 0, GPUS_MAX, pci_driver.name);
	if (r < 0) {
		printk(KERN_ERR "%s:cannot allocate major/minor range\n",
							pci_driver.name);
		goto class_destroy;
	}

	r = pci_register_driver(&pci_driver);
	if (r != 0) {
		printk(KERN_ERR "%s:cannot register PCI driver\n",
							pci_driver.name);
		goto chrdev_region_unregister;
	}
	return 0;

chrdev_region_unregister:
	unregister_chrdev_region(devt_region, GPUS_MAX);
	
class_destroy:
	class_destroy(class);
	return r;
}

static void cleanup(void)
{
	pci_unregister_driver(&pci_driver);
	unregister_chrdev_region(devt_region, GPUS_MAX);
	class_destroy(class);
	ida_destroy(&ida);
}

module_init(init);
module_exit(cleanup);

MODULE_AUTHOR("Sylvain Bertrand <digital.ragnarok@gmail.com>");
MODULE_DESCRIPTION("AMD southern island driver");
MODULE_LICENSE("GPL");
